#include "../include/PageLibPreprocessor.h"
#include "../include/simhash/cppjieba/Jieba.hpp"
#include "../include/simhash/Simhasher.hpp"
#include <algorithm>  


//this define can avoid some logs which you don't need to care about.
#define LOGGER_LEVEL LL_WARN 

using namespace simhash;

PageLibPreprocessor::PageLibPreprocessor(Configuration &pconf)//读取配置文件
{
    
}

void PageLibPreprocessor::doProcess()//执行预处理
{
    Configuration * pconf = Configuration::getInstance();
    auto config_map = pconf->getConfigMap();
    string path = config_map["xml_web"];

}

void PageLibPreprocessor::readInfoFromFile()//根据配置信息读取网页库和位置偏移库的内容
{

}
    
void PageLibPreprocessor::cutRedundantPages(SplitToolCppjieba &jieba, Configuration &pconf)//对冗余的网页进行去重
{
    //固定的传入词典路径 
    Simhasher simhasher("../include/simhash/cppjieba/dict/jieba.dict.utf8", "../include/simhash/cppjieba/dict/hmm_model.utf8",
                           "../include/simhash/cppjieba/dict/idf.utf8", "../include/simhash/cppjieba/dict/stop_words.utf8");

    size_t topN = 10;//选取权重最高的10个词
    uint64_t u64 = 0;
    
    vector<pair<string ,double> > res;
    
    unordered_map<string, uint64_t> hashRes;//文章和simhash值
    
    for(auto it = _pageLib.begin(); it != _pageLib.end(); ++it)
    {
        string s = it->getDoc();
        simhasher.extract(s, res, topN);//提取关键词和权重到res
        simhasher.make(s, topN, u64);//计算哈希值并存到u64中
        hashRes.insert(make_pair(s, u64));//将每篇文章id和对应的simhash值插入        
    }

    //根据simhash值去重
    for(auto it1 = hashRes.begin(); it1 != hashRes.end();){
        if (++it1 == hashRes.end())
        {
            break;
        }
        for(auto it2 = it1; it2 != hashRes.end(); ++it2){
            //isEuqal 相似返回0 不相似返回1
            if(!Simhasher::isEqual(it1->second, it2->second))
            {
                WebPage wp(it2->first, pconf, jieba);
                //删除所有值为wp的元素
                _pageLib.erase(std::remove(_pageLib.begin(), _pageLib.end(), wp), _pageLib.end());
            
            }
        }
    }
}


void PageLibPreprocessor::bulidInvertIndexTable()//创建倒排索引表
{

}

void PageLibPreprocessor::storeOnDisk()//将经过预处理之后的网页库、位置偏 移库和倒排索引表写回到磁盘上
{

}


